﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.IO;
using System.Net;
using System.Text.RegularExpressions;
using System.Collections;
using System.Threading;

namespace _1.SpyderWinform
{
    public partial class Form1 : Form
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        private int failCount = 0;
        public Form1()
        {
            InitializeComponent();

        }

        private void Form1_Load(object sender, EventArgs e)
        {
            textBoxTest.Text = "https://www.marxists.org/chinese/kropotkin/1886/11.htm";
        }

        private void button1_Click(object sender, EventArgs e)
        {
            panelFail.Controls.Clear();
            panelSuccess.Controls.Clear();
            string startUrl = textBoxUrl.Text;
            if(startUrl.IndexOf("https://")<0)startUrl = "https://"+textBoxUrl.Text;
            
            this.urls.Add(startUrl, false);

            this.Crawl();
        }


        private void Crawl()
        {
            Console.WriteLine("爬！");
            while (true)
            {
                //https://www.dy2018.com/1/?jdfwkey=sd60d
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if (current == null) break;

                Console.WriteLine("爬行" + current + "页");

                string html = DownLoad(current);
                urls[current] = true;
                count++;

                if(count==1)Parse(html,current);//只解析首页的html，只爬取起始网站上的网页
            }
            Console.WriteLine("爬行结束");
        }

        private string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                
                string html = webClient.DownloadString(url);
              
                string fileName = count.ToString()+".txt";
                File.WriteAllText(fileName, html, Encoding.UTF8);
                Label l1 = new Label();
                l1.Size = new Size(1500, 25);
                l1.Text = url;
                panelSuccess.Controls.Add(l1);
                l1.Location = new Point(0, count * 25);
                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                Label l1 = new Label();
                l1.Size = new Size(1500, 25);
                l1.Text = url+" 失败原因："+ex.Message;
                panelFail.Controls.Add(l1);
                l1.Location = new Point(0, failCount * 25);
                failCount += 1;
                return "";
            }
        }


        public void Parse(string html,string currentUrl)
        {
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
                if (strRef.IndexOf(".ico") >= 0 || strRef.IndexOf(".css") >= 0||strRef.IndexOf(".svg")>=0) continue;
                if (strRef.IndexOf(currentUrl) < 0) strRef = currentUrl + strRef;
                if(strRef.IndexOf("https://")<0)strRef = "https:" + strRef;
                if (strRef.Length == 0) continue;
                if (urls[strRef] == null) urls[strRef] = false;
            }
        }

        private void label1_Click(object sender, EventArgs e)
        {

        }

        private void label3_Click(object sender, EventArgs e)
        {

        }
    }
}
